package com.chance.cc.crawler.development.scripts.ebrun;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.*;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2020/11/30 9:33
 * @Description 亿邦动力网
 **/
public class EbrunCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(EbrunCrawlerScript.class);

    public static final String ebrunPrefixUrl = "https://www.ebrun.com";
    public static final String ebrunEntranceUrl = "https://www.ebrun.com/\\?eb=com_chan_nav_xddh";
    public static final String modelEntranceUrl = "https://www.ebrun.com/(retail|intelligent|b2b|fto|intl)/\\?eb=hp_home_nav_xzdh";
    public static final String modelArticleUrl = "https://www.ebrun.com/\\S+/\\d*";
    public static final String articleUrl = "https://www.ebrun.com/\\d{8}/\\d+.shtml";




    /**
     * 脚本domain定义
     *
     * @return
     */
    @Override
    public String domain() {
        return "ebrun";
    }

    /**
     * 进入脚本的正则列表
     */
    @Override
    public void initUrlRegulars() {
        addUrlRegular(ebrunEntranceUrl);
        addUrlRegular(modelEntranceUrl);
        addUrlRegular(modelArticleUrl);
        addUrlRegular(articleUrl);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     *
     * @param crawlerRequestRecord
     * @return
     */
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }


    /**
     * 解析链接方法
     *
     * @param crawlerRequestRecord
     * @param httpPage
     * @return
     */
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();

        String requestUrl = httpPage.getRequest().getUrl();

        if (requestUrl.matches(ebrunEntranceUrl)) {
            List<String> all = httpPage.getHtml().xpath("//ul[@class=\"menu\"]/li/a/@href").all();
            for (String url : all) {
                url = "https:" + url + "?eb=hp_home_nav_xzdh";
                if (url.matches(modelEntranceUrl)) {
                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .httpUrl(url)
                            .recordKey(url)
                            .notFilterRecord()
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    parsedLinks.add(itemRecord);
                }
            }
        } else if (requestUrl.matches(modelEntranceUrl) || requestUrl.matches(modelArticleUrl)) {
            Proxy proxy = new Proxy();
            proxy.setHost("http-dyn.abuyun.com");
            proxy.setPort(9020);
            proxy.setUsername("HL89Q19E86E2987D");
            proxy.setPassword("71F33D94CE5F7BF2");

            if (requestUrl.matches(modelEntranceUrl)) {
                //解析板块
                List<String> all = httpPage.getHtml().xpath("//div[@class=\"category pindao-category\"]/ul/li/a/@href").all();
                for (String url : all) {
                    url = ebrunPrefixUrl + url;
                    if (url.matches(modelArticleUrl)) {
                        CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .httpUrl(url)
                                .recordKey(url)
                                .notFilterRecord()
                                .releaseTime(System.currentTimeMillis())
                                .build();

                        parsedLinks.add(itemRecord);
                    }
                }
            }

            //下一页
            String nextPageUrl = httpPage.getHtml().xpath("//li[@class=\"pagination-next\"]/a/@href").get();
            if (StringUtils.isNotEmpty(nextPageUrl)) {
                CrawlerRequestRecord nextPageRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .needWashed(false)
                        .proxy(proxy)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                parsedLinks.add(nextPageRecord);
            }

            //解析文章列表
            List<Selectable> nodes = httpPage.getHtml().xpath("//div[@data-type=\"article\"]").nodes();
            for (Selectable node : nodes) {
                String itemUrl = node.xpath("./a/@href").get();
                if (StringUtils.isEmpty(itemUrl)) {
                    break;
                }
                itemUrl = itemUrl.matches(articleUrl) ? itemUrl : "https:" + itemUrl;
                String releaseTime = node.xpath(".//p[@class=\"time\"]/text()").get();
                if (StringUtils.isEmpty(itemUrl)) {
                    break;
                }
                try {
                    Long releaseTimeToLong = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .releaseTime(releaseTimeToLong)
//                            .needWashed(false)
                            .needParsed(false)
                            .resultLabelTag(article)
                            .proxy(proxy)
                            .build();

                    itemRecord.getHttpRequest().addHeader("Cookie"," _iebrunUu=1606699725904*419621466*994885162; _ebrunUu=1606699725912*211218581*958873868; UM_distinctid=17616c436b08d-0cddd79296ee92-393e5b09-144000-17616c436b12bf; sso_wwwebruncom_token=bq9mzjrguu0cg80kcw084g880; __adcookie=1606801558350*620232338*1166569519; _iebrunUv=1606801558431*1167449417*1636858562; _ebrunUv=1606801558435*1041836596*1861164386; CNZZDATA30003236=cnzz_eid%3D228302681-1606696747-%26ntime%3D1606799133; Hm_lvt_87daad7faca22f66ec178d201d855ddb=1606699727,1606801559; XSRF-TOKEN=eyJpdiI6IjZEMmFUSzJWQU5vTUNBU0h6QndMeEE9PSIsInZhbHVlIjoiRVFWXC9XYTQzeEhJN0hxaWRjWDloeWtaQzRsSHczZzBFMklVa05GakpwODlJcncxNXFnM3lua3ZqOW04bTlzVkdyRWpwaTlFRUd0ZEVBOEUyUmNvQzlBPT0iLCJtYWMiOiJkNzA2OGFkZjE4NGZhYzUxMjNmZTQ2YTZiNDQxZGM2YTM4OWI3MjI2NmY5ZDI2NTY5ZjRlODM2YmZlMjRlZjMyIn0%3D; laravel_session=eyJpdiI6IjBKcjFBM24xZDNuUXZKSzZCeFdjeGc9PSIsInZhbHVlIjoiRnRuXC9wUUozY21mK0Z4Y25RUnNwcFFDcURmMmkraGczN1h4UjJteU50NFwvak1laVFzOFhVTVNndVdGREJSdEY4TWlDSlVyNU0zVGVQalBXbVJtZUd3Zz09IiwibWFjIjoiOTZlNzkxNzYzOTY5Y2E5ODA2MjA4ZWM5NDgwNjg5OTBiZDcyZDE2NjQ1ZjJlYjNiMzBjZjFmYjZkMDZlY2ZhMiJ9; Hm_lpvt_87daad7faca22f66ec178d201d855ddb=1606802459; _Jo0OQK=4FD7C2417C0B9C41EAEDDC63A2D922388F9C39F7ECF225EA1311FE87945718DD652789D99F93ED9340BD2ED4BE30D3D21D1932B6DD9E4387CDB9D7183C54D77CD2186FA9B5075478CE81824A296BB017DFB1824A296BB017DFB4F3DA54CD054E0E8CC2983A57D1D9514GJ1Z1LA==");
                    itemRecord.getHttpRequest().addHeader("Host"," www.ebrun.com");
                    itemRecord.getHttpRequest().addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36");
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }
            }
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();

        boolean download = crawlerRecord.isDownload();
        int statusCode = page.getStatusCode();
        if(statusCode == 200 && download){
            if (crawlerResultTags.hasDataType(article)) {
                crawlerDataList.add(washArticle(crawlerRecord, page));
            }
        }else{
            log.warn("网页(“" + page.getRequest().getUrl()+ "”)下载错误，错误码：" + statusCode);
        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        String title = httpPage.getHtml().xpath("//h1[@class=\"post-text-title\"]/text()").get();

        List<String> authorAndSource = httpPage.getHtml().xpath("//span[@class=\"f-left\"]/text()").all();
        String author = "";
        String source = "";
        for (String data : authorAndSource) {
            if(data.matches("作者: \\S*")){
                author = data.substring(data.indexOf(":") + 1);
            }
            if(data.matches("来源: \\S*")){
                source = data.substring(data.indexOf(":") + 1);
            }
        }

        List<String> articleTextList = httpPage.getHtml().xpath("//div[@class=\"post-text\"]/p//text()").all();
        StringBuffer conents = new StringBuffer();
        for (int i = 0; i < articleTextList.size(); i++) {
            conents.append(StringEscapeUtils.unescapeHtml4(articleTextList.get(i))).append(" ");
        }

        String releaseTime = httpPage.getHtml().xpath("//span[@class=\"f-right\"]/text()").get();

        String itemUrl = httpPage.getRequest().getUrl();
        String articleKey = itemUrl.substring(itemUrl.lastIndexOf("/") + 1, itemUrl.lastIndexOf("."));

        CrawlerData crawlerData = null;
        try {
            crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime())
                    .addContentKV(Field_Content, conents.toString().trim())
                    .addContentKV(Field_Title, StringEscapeUtils.unescapeHtml4(title))
                    .addContentKV(Field_Source, source)
                    .addContentKV(Field_Author, author)
                    .build();
        } catch (ParseException e) {
            log.error("时间格式错误");
        }

        return crawlerData;

    }

    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    /**
     * 获取指定格式的字符串
     *
     * @param regx
     * @param input
     * @return
     */
    private static String getString(String regx, String input) {
        Pattern compile = Pattern.compile(regx);
        Matcher matcher = compile.matcher(input);
        List<String> urlList = new ArrayList<>();
        while (matcher.find()) {
            return matcher.group(0);
        }
        return null;
    }

}
